APPENDIX G 



// 
// 
// 
// 
// 
// 
// 
// 
// 
// 
// 
// 



FILE: Emb_b_w.cpp 

FUNCTIONALITY: class Embedded Training implementation 
PROGRAM: adding new files in database 
AUTHOR: A. CHRISTIAN TAHAN 
DATA FIRST VERSION: 02/15/00 



#include "Emb_b_w.h" 

//LogAdd: Return sum x + y on log scale, sum < LOGSMALL is floored to LZERO 
static t_real LogAddCt_real x, t_real y) 
{ 

t_real temp; 
if(x<y) 

{ 

temp = x; 
X = y; 
y = temp; 
} 

temp = y-x; 
ifCtemp<MINLOGEXP) 

if(x<LOGSMALL) 

return LOGZERO; 

else return x; 
else return x+log((t_real)1.0+ exp(temp)); 
} 

// * 

// Statistics Accumulators 

// * 

void StatisticsAccumulators: :Configure(const t_index num_states_of .symbol , 

const t_index num_mi xes_of .symbol , const 

t_index obs_size, 

const Boolean full_covs) 



{ 

t.index j, k; 
num_i stances = 0; 

num_states = num_states_of .symbol ; 
num_mixes = num_mi xes_of .symbol ; 

tran , Destroy _And_ReDim(num_states , num_states) ; 
occ . Destroy _And_ReDim(num_states) ; 
mu . Destroy _And_ReDim(num_states) ; 

f or( j=0 ; j<num_states ; 
{ 

mu[j] . Destroy_And_ReDimCnum_mixes) ; 

for(k=0; k<num_mixes; k++) 

mu[j][k] .Destroy_And_ReDim(obs_size); 

} 

if(full_covs) 
{ 

full_cov,Destroy_And_ReDim(num_states); 

f or( 3=0 ; j<num_states ; 
{ 

full_cov[j] .Destroy _And_ReDim(num_mixes); 
for(k=0; k<num_mixes; k++) 
{ 

full_cov[j] [k] . Destroy _And_ReDimCobs_size, 

obs.size); 

} 

} 

} 

else{ 

diag_va . Destroy _And_ReDimCnum_states) ; 
f or( j=0 ; j<num_states ; 
{ 

diag_va[j] . Destroy _And_ReDim(num_mixes) ; 
for(k=0; k<num_mixes; k++) 

diag_va[j]Ck] .Destroy _And_ReDim(obs_size) ; 

} 

} 

c . Destroy _And_ReDimCnum_states , num_mixes) ; 



return; 



} 

void StatisticsAccumulators: :Reset_Parameters() 
{ 

t_index j, k; 
num_istances = 0; 

tran.Set(0,0); 
occ.Set(0.0); 

f orC j=0 ; j<num_states ; 

for(k=0; k<num_mixes; k++) 
mu[j][k].Set(0.0); 

i f (f ul l.cov . Dim()>0) 

f or( j=0 ; j<num_states ; 

forCk=0; k<num_mixes; k++) 

f uU.covC j] [k] . Set(0 . 0) ; 
else for(j=0; j<num_states; 

forCk=0; k<num_mixes; k++) 

di ag.va [ j] [k] . Set(0 . 0) ; 

c.Set(0.0); 

return; 

} 



// 

// REESTIMATE MODELS PARAMETERS 

// 

// UNTIL THE DESIRED CONVERGENCE VALUE IS REACHED 

// 

Boolean ModelsSimultaneousTraining : : Symbol_Models_Optimisation() 
{ 

t_real average.likelyhood; 
t_index iteration_counter = 0, i; 

of St ream file; 

f i I e . open( " I i kely hood . i nt " ) ; 
file.closeO; 

if(load_accs) 

Update_Models(); 



else{ 

mstat«"Start parameter reestimation cycle."; 



{ 

f or(i=0 ; i<HMM_accs . Dim() ; 

HMM_accs[i] .Reset_Parameters() ; 
ReEstimate_Parameters(average„li kelyhood , dbase) ; 
Update_Models(); 
iteration_counter++; 

f i le . open( " I i kely hood . i nt " , i os : : app) ; 
file«"Average likelyhood at iteration 

«" : "«average_likelyhood«"\n" ; 
file.closeO; 

Store_Iteration_Model(iteration_counter); 
dbase. RestartO; 

} while(iteration_counter<max_num_iteration); 

mstat«"Training set likelyhood: "«average_likelyhood; 
} 

file.closeO; 

if(store_accs) 

Store_Statistic_Accs(accs_file); 
return (Boolean)TRUE; 

} 

void ModelsSimultaneousTraining: :Store_HMM_Models() 
{ 

t_index symb; 
String f_name; 

for(symb=0; symb<HMM_defs.Dim(); symb++) 

HMM_defs[symb] .Write(models_file_output); 

return; 

}; 

void ModelsSimultaneousTraining: :Store_Iteration_Model(t_index iteration) 
{ 

t_index symb; 
of St ream file; 



do 



"«iteration_counter 



String f_name; 

f_name«"iteration_"«iteration«" .emb"; 
f i le . open(f _name) ; 

Write_Header.Of_File.Model(f_name, dbase.Snd_Type(), 
dbase . Label_Type() , dbase . Db.Fi le_Li st_Name 
() , dbase . Wi ndow_Lenght() , 

dbase. Window.OverlapC), 39, FALSE); 

file.closeO; 

for(symb=0; symb<HMM_defs.DimC); symb++) 
{ 

HMM„defs[symb] .Write(f_name); 
} 

return; 
} 



// Init_Emb_Train: initialise dbase, training structures and load models, 
void ModelsSimultaneousTraining: :Configure(const String& config.file) 
{ 

ConfigFile conf; 

t.index symb_num, position; 

features . Conf i gure(conf i g_f i le) ; 
dbase . Conf igure(conf ig_f i le , TRUE) ; 
symb_num=dbase . Get_Num_Of _Symbols() ; 
conf . Open_Fi le(conf i g_f i le) ; 

conf .Get_String_OptC'ModelsSimultaneousTraining" , "modelsfilename" , 
models_f ile_input) ; 

max_num_iteration = conf .Get_Unsigned_Opt 
("ModelsSimultaneousTraining" , "MaxNumlteration") ; 

reest_means=conf .Get_Boolean_Opt("ModelsSimultaneousTraining" , 
"ReestimateMeans") ; 

reest_variances = conf .Get_Boolean_Opt 
("ModelsSimultaneousTraining" , "ReestimateVariances") ; 

reest_weights = conf .Get_Boolean_Opt 
("ModelsSimultaneousTraining" , "ReestimateWeights") ; 

reest.transitions = conf .Get_Boolean_Opt 
("ModelsSimultaneousTraining" , "ReestimateTransitions"); 

min_istance_number = conf .Get_Unsigned_Opt 



("ModelsSimultaneousTraining" , "MinlstanceNumber") ; 

pruning.threshold = conf . Get_Real_OptC'ModelsSimultaneousTraining" , 
"PruningThreshold") ; 

min_var_value = conf .Get_Real_Opt("ModelsSimultaneousTraining" , 

"Mi nimunVarianceValuelf Diagonal"); 

const^to_add_min_var= conf .Get„Real_Opt 
("ModelsSimultaneousTraining" , "ConstToAddMinVar") ; 

load.accs = conf .Get.Boolean_Opt("ModelsSimultaneousTraining", 
"LoadAccumulators") ; 

if(load_accs) 

conf .Get_String_OptC"ModelsSimultaneousTraining", 
"AccsList", list.file); 

store_accs = conf .Get_Boolean_Opt("ModelsSimultaneousTraining" , 
"StoreAccumulators") ; 

if(store_accs) 

conf .Get_String_Opt("ModelsSimultaneousTraining" , 
"AccsFile", accs.file); 

models_f ile_input . Is_SubString_Inside(" . " , position) ; 
models_file_output .Destroy_And_ReDim(position); 
models.f i le_output . Take(models_f i le_i nput , 0 , posi tion) ; 
models_f ile_output«"emb" ; 



HMM_def s . Destroy _And_ReDimCsymb_num) ; 
Load_Models_Parameters() ; 
if(load_accs) 
{ 

ifstream file_list; 
String file_name; 



file_list.open(list_file, ios: :inl ios: rnocreate); 
if(file_list.fail()) 

merr«"Cannot open file of list of statistics 

accumulators" ; 



whi le( ! f i le_list . eof ()) 
{ 

f i I e_l i s t»f i I e_name ; 

if(file_list.eof() AND file_name[0]==EOF) 
return; 

Load_Statistic_Accs(f ile_name) ; 
} 

} 



return; 

}; 



// * 

// REESTIMATE PARAMETRS 

* 

// * 



t_real ModelsSimultaneousTraining: :Sequences_Total_LProbCconst VetDoubleS 
sequences.lprob) 
{ 

t_real temp; 
t_index i; 

temp = sequences_lprob[0] ; 

f or(i=l ; i<sequences„lprob . Dim() ; 

temp+=sequences_lprob[i] ; 

return temp; 

}; 

void ModelsSimultaneousTraining: :ReEstimate_Parameters(t_real& 
t rai ni ng_set_l i kelyhood , 

DbaseVoc& dbase) 

{ 

Boolean not_end_of_dbase, is„new_file; 



t_index i, file.counter ; 
VetDouble vetsmp; 
t_string_list label_list; 
t_real sequence_lprob; 
VetDouble sequences_lprob; 

static VetDouble observation_vet ; 
VetDoubleList pred_list; 
T = 0; 



not_end_of .dbase = (Boolean)TRUE; 

pred.list. Destroy _And_ReDim(features.Max_Delta_Feature_Order()); 



whi leCnot_end_of _dbase) 
{ 



// indicating in "is_new_ file" if it is the first frame of 



a 

// new sentence and the condition "not_end_of_dbase" 
not.end_of _dbase=dbase . Get_Sequential_Vet(vetsmp , 

is_new_file); 



if( (T!=0) AND (is_new_file==TRUE OR 
not_end_of_dbase==FALSE ) ) 

//i.e. if an file has been read 
{ 

// TRAINING STEP: apply forward-backward procedure 
// to given file and update parameter 

accumulators 



Assert(T === file.Dim()); 
sequence.lprob = Update_Parameters_With_New_file(file, 

label_idxes_list) ; 
sequences„lprob.Append(sequence_lprob); 
file.ResetO; 

cout«"f i le_num : "«sequences_lprob . Dim()« ' \t ' ; 

cout«"num frames: "«T«endl; 

T=0; 

} 

if ( not_end_of_dbase ) 
{ 

if(is„new„file) 
{ 

dbase . Get_Label_List_Of _Actual_f i le 

(label_idxes_list) ; 

Q = label_idxes_list.Dim(); 
f or(i=0 ; i <pred_list . Dim() ; i++) 

// just a little trick: starting 

frames initialized with 

// the value of the first frame: in 

order to have a starting value 

pred_list[i] = vetsmp; 
features . Get_Previous_Frames_Inf o 

Cpred.list, dbase. Smp_Rate()); 

} 

features . Extract(observation_vet , vetsmp , 

dbase . Smp_Rate()) ; 

// memorize in file the samples of the next file 
// of training data 



fx le . Append(observation_vet) ; 
T++; 

} 



} // end while 
file_counter = sequences_lprob.Dim(); 

training_set_likelyhood = Sequences_Total_LProb(sequences_lprob)/ 
(t.real)file_counter; 

return; 

}; 



t„real ModelsSimultaneousTraining: :Update_Parameters_With_New_file 

(const Bi_D_List& file, const VetULong& 

label.idxes_list) 
{ 

t_index q, t, startq, endq; 
t.real file_lprob; 

VetDouble obs; // observation vector 

VetULong top_label; // top of pruning beam 

VetULong bottom_label ; // bottom of pruning beam 



file_lprob = Compute_Beta(top_label , bottom_label , file, 
label_idxes_list) ; 

• if (file.lprob > LOGZERO) // if models fit current- file 

{ 

//alfa computation and parameters updating 
//performed only if model fits actual file 
Init_Alpha(label_idxes_list); , 

//update occurrence counters of the HMMs 
for (q=0;q<Q;q++) 

HMM_accs[label_idxes_list[q]] .num_istances++; 
//for all frames 
for (t=0;t<T;t++) 
{ 

Get_Observation(obs, file, t); 
if (t>0) 

Compute_Alpha_At_Time_t(startq , endq , 

file.lprob, top.label, 



bottom_label , t, label_idxes_list); 

else startq = endq = 0; 

// update parameters only on a significant sequence 
// ( starting at label no. startq and ending at 

label endq) 

// of labels inside the actual file 
for (q=startq;q<=endq;q++) 
{ 

Update_Occourrence_Counter(HMM_accs 
[label_idxes_list[q]] , t,q, file.lprob); 

if (reest^means OR reest_variances OR 

reest.weights) 

// HMM_defs[label_list[q]] is the 

hmm 

// of the q-th labelof the file 
Up_Mix„Parms(HMM_defs 
[label_idxes_list[q]] , HMM_accs[label_idxes_list[q]] , 

q, t, obs, 

file_lprob) ; 

if Creest_transitions) 

Up_Tran_Parms(HMM_defs 
[label_idxes_list[q]] , HMM_accs[label_idxes_list[q]] , 

file_lprob, q, Q, 

t, T); 

} // endfor q 
} // endfor t 
}// end if 

return file_lprob; 
} 



// InitPruneStats: initialise pruning stats 
void ModelsSimultaneousTraining: : Init_Prune_Stats() 
{ 

maxBeamWidth = 0; 
maxAlphaBeta = LOGZERO; 
minAlphaBeta = 1.0; 
} 



// CheckPruning: record peak alfa.beta product and position 

void ModelsSimultaneousTraining: :CheckPruningCt_index t, const t_index 



beam.top, const t.index beam_bottom, 
EmbCodebookSt act_hmm) 

{ 

t^signed margin; 

t_index i, q, bestq, besti, Nq; 

t_real 1, maxL; 

bestq = besti =0; 
maxL = LOGZERO; 

for (q=beam_bottom ; q<=beam_top ; q++) 
{ 

Nq = act_hmm.num_states; 
for (i=l;i<Nq-l;i++) 
{ 

if((l-act_alfa[q][i] + beta[q] [t] [i]) > maxL) 
{ 

bestq = q; 
besti = i; 
maxL=l; 
} 

} 

} 

if (maxL > maxAlphaBeta) 

maxAlphaBeta = maxL; 
if (maxL < minAlphaBeta) 

minAlphaBeta = maxL; 
margin = beam_top-beam_bottom+l; 
if (margin>maxBeamWidth) 

maxBeamWidth = margin; 

} 

// Createlnsts: create array of hmm instances indexes 

void ModelsSimultaneousTraining: :Create_Insts(VetULong& label_idxes_list, 

const t_string_list& label_list, DbaseVoc& dbase) 
{ 

t_index q; 

label_idxes_li St . Destroy _And_ReDim(label_li st . Dim()) ; 
for (q=0;q<label_list.DimC);q++) 

label_idxes.list[q] = dbase. Translate_Symbol(label_list 

[q]); 



return; 
} 



// SetBeta: allocate and calculate beta and otprob matrices 

t_real ModelsSimultaneousTraining: :Compute_Beta(VetULong& qHi, VetULongS 

qLo, 

const Bi_D_List& 

whole_file, const VetULongS label_idxes_list) 
{ 

t_index t,q; 

t.index i, j, Nq, q_at_gMax, startq, endq, last_q; 
VetDouble bqt,bqtl,outprob,maxP; 
t_real x, a, y, gMax, IMax; 
EmbCodebook *act_HMM; 
t_real pr; 

// Create Storage Space - excluding actual data arrays 

// storage for min and max q values 
qHi . Destroy.And_ReDim(T) ; 
qLo . Destroy_And_ReDim(T) ; 

// dimensionate beta and obs_lprob 
beta . Destroy _And_ReDim(Q) ; 
obs.lprob . Destroy _And.ReDim(Q) ; 
for(q=0;q<Q;q++) 

{ 

beta [q] . Destroy _And_ReDim(T) ; 
obs_lprob[q] .Destroy _And_ReDim(T); 
} 

maxP.Destroy_And_ReDim(Q); // for calculating 

beam width 

act.HMM = &HMM_defs[label_idxes_list[Q-l]]; 
Nq = act_HMM->num_states ; 
beta[Q-l] [T-1] .Destroy_And_ReDimCNq); 
beta [Q- 1] [T- 1] [Nq - 1] =0 . 0 ; 

for (i=l;i<Nq-l;i++) 

beta[Q-l][T-l][i3 = act„HMM->trans_mat[i][Nq-l] ; 

betaCQ-1] [T-1] [0]=LOGZERO; 
qHi[T-l] = qLo[T-l] = Q-1; 

Compute.Obs_LProbs(whole_file[T-l], T-1, qHi [T-1], 



qLo[T-l], label_idxes_list); 
Assert (T>=2); 



for (t-T-2;t!=(t_index)(-l);t— ) 
{ 

gMax = LOGZERO; // max value of beta at time t 
if (t>=qHi [t+1]) startq=qHi [t+1] ; 
else startq = t; 

if (0==qLo[t+l]) endq = 0; 
else endq = qLo[t+l]-l; 

Assert(startq>=endq) ; 
for (q=startq;q!=(t_index)(endq-l);q--) 
{ 

IMax = LOGZERO; // max value of beta in model q 

act.HMM = &HMM_defsClabel_idxes_listCq]]; 

Nq = act_HMM->num_states ; 

// create vec for beta vals 

beta [q] [t] . Destroy _And_ReDim(Nq) ; 

outprob = obs_lprobCq][t+l] ; 

if (q==:startq) beta[q][t][Nq-l]= LOGZERO; 
else beta[q][t][Nq-l]= beta[q+l][t][0] ; 

Assert(Nq>=2); 

for (i=Nq-2 ; i ! =(t_index)(-l) ; i 
{ 

x = act_HMM->trans_mat[i][Nq-l] + beta[q]Ct][Nq-l] ; 

if (q>=qLo[t+l] AND q<=qHi[t+l]) 
for (j=l;j<Nq-l;j++) 
{ 

a = act_HMM->trans_mat[i][j] ; 

y - betaCq][t+l][j]; 
if (a>LOGSMALL AND 

y>LOGSMALL) 

X = LogAdd(x,a 

+outprob[j]+y); 

} // endfor j 

beta[q][t][i] = x; 
if (x>lMax) IMax = x; 
if (x>gMax) 
{ 

gMax = x; 
q_at_gMax = q; 
} 

} // endfor i 
maxP[q] = IMax; 



} // endfor q 



last_q = endq; 

while (gMax-maxP[startq] > pruning_threshold) 

startq-=l; // lower startq till threshold 

reached 

qHi[t] = startq; 

while ( ((gMax-maxP[endq]) > pruning_threshold) AND endq<t) 

endq+=l; // raise endq till thresh reached 

qLo[t] = endq; 

Compute_Obs_LProbs(whole^file[t], t, qHi[t], qLo[t], 
label_idxes„list); 

} // endfor t 

// compute total probability pr 
pr = LOGZERO; 

outprob = obs_lprob[0][0] ; 
for (j=l; j<Nq-l;j++) 
{ 

a = act_HMM->trans_mat[0][j]; 

y = beta[last.q][0][j]; 

if ( (a>LOGSMALL) AND (y>LOGSMALL) ) 

pr = LogAdd(pr,a+outprob[j]+y); 

} 

if (LOGZERO >= pr) 
{ 

mwarn«"Prune threshold = "«pruning_threshold«" too 

small."; 

return pr; 
} 

return pr; 
} 

// Setotprob: allocate and calculate otprob matrix at time t 
void ModelsSimultaneousTraining: :Compute_Obs_LProbs(const VetDoubleS obs, 

const t.index t, const t_index beam_top, 
const t_index beam_bottom, const VetULong& 

label_idxes_list) 
{ 

t_signed q; 

t_index j, Nq, endq; 



VetDouble temp_dvet ; 
EmbCodebook *act_HMM; 



if (0==beam_bottom) endq = 0; 
else endq = beam_bottom-l; 

for (q=beam_top; q>=(t_signed)endq; q--) 
{ 

act_HMM = &HMM_defs[label_idxes_Ust[q]]; 
Nq = act_HMM->num_states ; 

obs_lprob[q] [t] .Destroy_And_ReDim(Nq-l) ; 
for (j=l;j<Nq-l;j++) 

obs_lprob[q][t][j]=(*act_HMM)[j-l].Obs_LProb(obs); 

} 

return ; 
} 



// 
// 

// ALPHA MATRIX 

// 
// 

// InitAlpha: allocate and initialise olfa columns for time t=l 
void ModelsSimultaneousTraining: :Init_Alpha(const VetULong& 
label_idxes_list) 
{ 

t_index i,j,q, Nq; 

VetDouble aq,outprob; 
EmbCodebook *act_HMM; 
t_real x; 

// Create Storage Space - two columns 
act.alf a . Destroy _And_ReDim(Q) ; 
prev.alf a . Destroy„And_ReDim(Q) ; 
for(q=0;q<Q;q++) 
{ 

Nq = HMM_defs[label_idxes.list[q]] .num_states; 
act_alfa[q] . Destroy _And_ReDim(Nq) ; 
act_alfa[q].Set(LOGZERO); // initialize act.alfa to 

LOGZERO 



prev_alf a [q] . Destroy _And_ReDim(Nq) ; 
} 

// Calculate prev_alfa (alfat) values for t=0 



act.HMM = &HMM_defs[label_idxes_list[0]]; 
Nq = act_HMM->num_states; 
act_alfa[0][0] = LOGZERO; 

if(obs_lprob[0][0] .Dim()==0) 

merr«"No observation prob matrix at time t:=0."; 

for (j=l;j<Nq-l;j++) 

if (act_HMM->trans_mat[0] [j]>LOGSMALL) 

act_alfa[0][j] = obs_lprob[0][0][j]+act_HMM- 

>trans_mat[0][j] ; 

else act_alfa[0][j] = LOGZERO; 

X = LOGZERO; 

for (i=l;i<Nq-l;i++) 

if (act_HMM->trans_mat[i] [Nq-1]>L0GSMALL) 

X = LogAddCx, act_alfa[0] [i] + act_HMM->trans_mat 

[i][Nq-l]); 

act_alfa[l][0] = x; 
Zero_Alpha(l,Q-l, label_idxes_list); 

return; 
} 



// StepAlpha: calculate alfat column for time t and return 
// forward beam limits in startq and endq 
// for first opbservation startq = endq=0 

// startq and endq must be passed by reference to update their value 
void ModelsSimultaneousTraining: :Compute_Alpha_At_Time_t(t_index& startq, 
t_index& endq, 

const t_real pr, const 

VetULongS qHi, const VetULong& qLo, 

const t_index t, const 

VetULong& label„idxes_list) 
{ 

Bi_D_List tmp; 

EmbCodebook *act.HMM; 
t_index i , j ,q,Nq; 
t_real a,x,y; 

startq = qLo[t-l]; 



while (pr - MaxModelProbCstartq, t-1, label_idxes_list)>MINMODELPROB) 

startq+=l; 
if (startq<qLo[t]) 

startq = qLo[t]; 
endq = qHi[t-l]; 

while (pr - MaxModelProb(endq, t-1, label_idxes_list)>MINMODELPROB) 

endq-=l; 
if (endq>qHi[t]) 

endq = qHi[t]; 

tmp = prev_alfa; prev_alfa = act_alfa; act_alfa = tmp; 
act_alfa[0][0] = LOGZERO; 

if (startq>0) Zero_Alpha(0, startq-1, label_idxes_list); 

// at any time the t-th column of alpha matrix is calculated only 

for that 

// model states in the range [stratq; endq]; out of that range the 

alfa 

// values are not meaningful 

for (q = startq; q <= endq; q++) 
{ 

act_HMM = &HMM_defs[label_idxes_list[q]]; 

Nq = act_HMM->num_states; 

if (obs_lprob[q][t].Dim() 0) 

merr«"Bug outprob NULL at time t in StepAlphfci, " ; 

for (j=l; j<Nq-l;j++) 
{ 

X = LOGZERO; 
for (i=0;i<Nq-l;i++) 
{ 

a=act_HMM->trans_mat[i] [j] ; 
y = prev_alfa[q][i] ; 
if(a>LOGSMALL AND y>LOGSMALL) 
X = LogAdd(x, y + a); 

} 

act_alfa[q][j] = x + obs_lprob[q] [t] [j] ; 
} 

X = LOGZERO; 
for (i^l;i<Nq-l;i++) 
{ 



a=(act_HMM->trans_mat[i] [Nq-1]) ; 
y = act_alfa[q][i]; 
if(a>LOGSMALL AND y>LOGSMALL) 
X = LogAdd(x,y+a); 

} 

act_alfaCq][Nq-l] = x; 
if (q<Q-l) 

act_alfa[q+l][0] = x; 

} 

if (endq<Q-l) Zero_Alpha(endq+l,Q-l, label_idxes_list); 

return; 
} 

// ZeroAlpha: zero alpha's of given models 

void ModelsSimultaneousTraining: :Zero_Alpha(const t_index qlo, const 
t_index qhi, 

const VetULong& label_idxes_list) 

{ 

t_index j, q, Nq; 

// qlo model - dont erase carry over 

Nq = HMM_defs[label_idxes_list[qlo]] .num_states; 

for (j=l;j<Nq;j++) 

act_alfa[qlo][j] = LOGZERO; 
for (q=qlo+l;q<=qhi ;q++) 

{ 

// general case - all states LZERO 

Nq = HMM_defs[label_idxes_list[q]] .num_states; 

for Cj=0;j<Nq;j++) 

act_alfa[q][j] = LOGZERO; 

} 

if (qhi<Q-l) 

act_alfa[qhi+l][0] = LOGZERO; 

return; 
} 



// 4c4c9ic***«9ic9ic****4c gjsjQ y^LPHA MATRIX ************************ 

// GetOutVec: Get observation vector obs 

void ModelsSimultaneousTraining: :Get_Observation(VetDouble& obs, 

const Bi_D_List& whole_file, const t_index 



t) 

{ 

obs = whole_file[t] ; 

return; 

} 

// MaxModelProb: Calc max probability of being in model q at 
// time t, return LOGZERO if cannot do so 

t_real ModelsSimultaneousTraining: :MaxModelProb(const t_index q, const 
t_index t, 

const VetULongSe label_idxes_list) 

{ 

t_real maxP,x; 
t_index Nq,i; 

maxP = LOGZERO; 

ifCq<beta.Dim()) 
{ 

Nq = HMM.defs[label_idxes_list[q]] .num.states; 

if Cbeta[q][t].DimO!=0) 
{ 

for (i=0;i<Nq-l;i++) 

if C(x-act_alfa[q][i]+beta[q][t][i]) > 

maxP) 

maxP = x; 
} 

} 

return maxP; 
} 



// 

// FIRST (HIGHER) LEVEL FUNCTION 

// Update_Models 

// 

void ModelsSimultaneousTraining: :Update_Models() 

.{ 

t_index h, n; 



// for each dbase label-> i.e. for each file 
for (h=0;h<HMM_defs.Dim();h++) 
{ 

n = HMM_accs[h] .num_i stances; 

if (n < niin_istance_number ) 
{ 

mwarn«" Insufficient training examples for model."; 
mwarn«"Model "«h«" copied: only "«n«" examples"; 
} 

else{ 

Update_Model(HMM_defs[h] , HMM_accs[h]) ; 
mstat«"Model "«h«" updated with "«n«" examples 

\n"; 

} 

} 

return; 
} 



// 

// SECOND LEVEL FUNCTIONS 

// Update_Models 

// 

void ModelsSimultaneousTraining: :Up_Mix_Parms(EmbCodebook& hmm, 
Stati sti csAccumulatorsS acc , 

const t_index q, const t_index t, 

const VetDouble& obs, const t_real pr) 
{ 

t_index i , j ,k,kk,m,N, obs_size; 

VetDouble aqtl; 

t_real log_gamma, initx; 

t_real gamma, w, zmean,zmeanl,zmean2; 

VetDouble mean; 

N = hmm.num.states; 



if(t==0) aqtl.ResetO; 
else aqtl=prev_alfa[q] ; 



for (j=l;j<N-l; 
{ 

if ((t==0) AND (q==0)) 

initx = hmm.trans_mat[0][j] ; 
else if ((t==0) AND Cq!=0)) // 

shouldn't happen 

initx = LOGZERO; 

else{ 

initx = LOGZERO; 
for (i=0;i<N-l;i++) 

initx = LogAdd(initx, aqtl[i] + 

hmm.trans_mat[i][j]); 

} 

if (initx>LOGSMALL) 

for (m=0;m<hmm[j-l] .Dim();m++) 
{ 

w = hmm[j-l][m] .weight; 
if (w>LMINMIX) 
{ 

mean = hmni[j-l] [m] .mean; 
log_gamma = initx + w + beta[q][t] 

[j]; 



[j-l] .Mix_Obs_LProb(obs,m)-pr; 



(log_gamma) ; 



if (log_gamma>LOGSMALL) 
{ 

log_gamma += hmm 

if (log_gamma>MINEXPARG) 
{ 

gamma = exp 



[j][m].DimO; 

(k=0 ; k<obs„size ; k++) 

(reest_means) 

[j]M [•<]+= obs[k]*gamma; 

(reest_variances) 



obs_size = acc.mu 
for 

{ 

if 



if 



acc.mu 



{ 

if 



( ! hmm . f ull_covariance) 
{ 

zmean=obs[k]-mean[k] ; 

acc.diag_va[j][m] [k] += (zmean*zmean*ganima); 
}else { 

zmeanl = obs[k]-mean[k] ; 
for (kk=k;kk<obs_size;kk++) 
{ 

zmeanZ = obs[kk]-mean[kk] ; 

acc . f ull_cov[j] [m] [k] [kk]+= (zmeanl*zmean2*ganima) ; 
} // endfor kk 

} // endif (!hmm.FuU_Cov()) 

} 

// endif (reest_variances) 

} // endfor k 

if (reest_weights) 

acc.c[j][m] 



+=gamma ; 

(log_gamma>MINEXPARG) 



} 
} 

} // endfor m 

} // endfor j 
return; 
} 



} // if 

// endif (x>LOGSMALL) 



// UpTranParms: update the transition counters of given acc 

void ModelsSimultaneousTraining: :Up_Tran_Parms(const EmbCodebook& hmm, 

StatisticsAccumulators& acc, 

const t_real pr, const t_index q, const t_index Q, 

const t_index t, const t_index T) 



{ 

t_index i,j,N; 

t_real gamma; 
VetDouble bqtl; 

N = hmm . num_states ; 

if(t==T-l) 
{ 

bqtl.ResetO; 
} 

else bqtl = beta[q][t+l] ; 
if Ct<T-l) 

{ 

for (i=0;i<N-l;i++) 
{ 



for (j=l;j<N-l;j++) 
{ 

if (i==0 AND q==0) 
{ 



[q][t][j]-pr; 

acc.tran[i][j] += exp(gamma); 



if (t==0) 
{ 

gamma = act_alfa[q][j]+beta 

if (gamma>MINEXPARG) 

} 

} 



else{ 

if (bqtl.DimO != 0) 
{ 

gamma = act_alfa[q][i] 
+hmm.trans_mat[i][j]+obs_lprob[q][t+l] [j]+bqtl[j]-pr ; 

if (gamma>MINEXPARG) 

acc.tran[i][j] += exp(gamma); 

} 

} // endif (i==0 AND q==0) 

} // endfor j 
if (q<Q-l) 
{ 

gamma = act_alfa[q][i]+ hmm.trans_mat[i] 

[N-l]+betaCq] [t] [N-1] -pr ; 

if (gamma>MINEXPARG) acc.tran[i] [N-1] += 

exp(gamma) ; 



} 

} // endfor i 

} 

else{ 

if (q==Q-l) 

for Ci=l;'i-<N-l;i++) 
{ 

gamma = act_alfa[q][i]+beta[q][t][i]-pr; 
if (gamma>MINEXPARG) acc.tran[i] [N-1] += 

exp(gamma); 

} 

} 

return; 
} 



// UpOccCount: update the occupation counters of given acc 
void ModelsSimultaneousTraining: :Update„Occourrence_Counter 
(StatisticsAccumulatorsS acc, 

const t.index t, const t_index q, const t_real pr) 
{ 

t_index i,N; 
t_real gamma; 

N = acc.occ.DimO; 
if (q==0) 
{ 

if (t==0) acc.occ[0]+=l; 
} 

else{ 

gamma = act_alfa[q][0]+beta[q][t][0]-pr; 
if (gamma>MINEXPARG) acc.occ[0] += exp(gamma); 
for (i=l;i<N-l;i++) 
{ 

gamma = act_alfa[q][i]+beta[q][t][i]-pr; 

if (gamma>MINEXPARG) acc.occ[i] += exp(gamma); 

} 
} 

return; 
} 



// 



Model Update 



void ModelsSimultaneousTraining: :Update_Model(EmbCodebook& hmm, 
StatisticsAccumulators& acc) 
{ 

t.index i, j, k, kk, m, N, obs.size; 

t_real new_aij, new_mix_weight , new.var, occi ,c.im,cfloor ; 

VetDouble mean; 

MatrixOf Double covariance; 

VetDouble mu_im,va_im_var,va_im_inv_k; 

MatrixOf Double va_im_inv; 

cfloor = MINMIX; 
N = hmm,num_states; 

if (reest_transitions) 
{ 

hmm . t rans_mat . Set(LOGZERO) ; 
for (i=0;i<N-l;i++) 
{ 

occi = acc.occ[i] ; 
if (occi > 0.0) 

for Cj=l;j<N;j++) 
{ 

new_aij = acc. tran[i] [j]/occi ; 
if(new_aij>MINLOGARG) 

hmm,trans_mat[i][j] = log 

(new_ai j); 

else hmm.trans_matCi][j] = LOGZERO; 
} 

else mwarn«"Model "«hmm.file«" state "«i«" 

never occupied."; 

} // endfor i 
} // endif (reest.transitions) 

if (reest_means OR reest_variances OR reest.weights) 
{ 

// for each effective spectral state 
for (i=l;i<N-l;i++) 
{ 

occi=acc.occ[i] ; 
if (occi > 0.0) 
{ 

M= hmm[i-l] .Dim(); 
// for each gaussian of i-th state 
for (m=0;m<M;m++) 
{ 



if (M==l) 

c_im = occi; 
else c_im = acc.c[i][m] ; 
new_mix_weight = c.im/occi; 
if (reest_weights) 

{ 

if (new_mix_weight > 1,0) 
{ 

if (new_nfiix_weight 



> 1,001) 
serious! 

too big (new_mix_weight = "«new_mix_weight 
«") in model n. "«hmm.file«" state "«i 
«", mix "«m; 
new_mix_weight =1.0; 



// this is 
mwarn«"Mix 



(new_mix_weight<MINLOGARG) 
= LOGZERO; 

log(new_mix.weight) ; 



acc . mu [i] [m]/(t_real)c_im ; 



hmm . f u 1 1 _cova r i a n ce) 

hmm[i -1] [m] . diag.inv.cov . Dim() ; 
(k=0 ; k<obs_si ze ; k++) 



if 

hmm[i-l] [m] .weight 
else hmm[i-l][m] .weight = 
} 

if Cnew_mix_weight >= cfloor) 
{ 

if (reest_means) 

hmm[i-l][m] .mean = 



if (reest_variances) 
{ 

if (! 

{ 

obs.size = 



for 



new.var = acc.diag_va[i][m][k]/c_im; 

if 

(new_var > min_var_value) 

hmm[i-l][m] .diag_inv_cov[k][k]= new_var; 
else hmm[i-l][m] .diag_inv_cov[k][k]= const_to_add_min_var+new_var; 

} 

// endfork 

} 

else{ 

obs_size = 

hmm[i -1] [m] . inv_cov.Dim() ; 

for 

Ck=0 ; k<obs.si ze ; k++) 

for 

(kk=k ; kk<obs_size ; kk++) 
{ 

new_var = acc.full_cov[i][m][k][kk]/c_im; 
if(k == kk AND new_var < niin_var_value) 

new.var = min_var_value; 
hmm [i - 1] [m] , i nv_cov [k] [kk] 

= hmm[i-l][m] .inv_cov[kk][k] = new.var; 

} 

} 

ifC!hmm[i-l] 

[m] .Compute_G_Const()) 

mwarn«"Invalid inverse matrix in file: "«hmm.file 

«' 

state: "«i«" gauss: "«m; 

} // endif 

(reest_variances) 

} // endif (new_mix_weight 

>= cfloor) 

} // endfor m 
} // endif (occi>0) 
else mwarn«"Model "« hmm. file «" state" « i« " 



never occupied. \n"; 

} // end for i 
} //endif (reest_means OR reest_variances OR 

reest_mixtures) 
return; 
} 



// He******:*********** UPDATE MODELS ******************* 



void ModelsSimultaneousTraining: :Store_Statistic_Accs(const String& 
accs_file) 
{ 

t_index i , j ,h,k,z,Nh,Mh; 
t_index obs_size; 
of stream file; 

f i le . openCaccs_f i le) ; 

f i le . preci si on(OUTPUT_SIZE) ; 

f or(h=0 ; h<HMM_accs . Dim() ; h++) 
{ 

Nh=HMM_accs[h] .num_states; 
Mh=HMM_accs [h] . nuni_mixes ; 

file«"file: "«h«"\n\n"; 
file«"num_istances= "«HMM_accs[h] .nuni_istances«"\n" ; 
file«"num_states= "«Nh«"\n" ; 
f ile«"nutti_mixes= "«Mh«"\n\n" ; 
file«"tran:\n"; 

for(i=0;i<Nh-l;i++) 
{ 

for(j=l; j<Nh; 
file«HMM_accs[h] .tran[i][j]«" "; 
file«"\n"; 
} 

file«"\nocc: "; 
for(i=0;i<Nh-l;i++) 

file«HMM_accs[h].occ[i]«" "; 

obs_size=HMM_accs[h] .mu[0] [0] .Dim(); 



file«"\n\nmu:\n" ; 
for(i=l;i<Nh-l;i++) 

f o r( j =0 ; j <Mh ; j ++) 
{ 

f or(k=0 ; l«obs_size ; k++) 

f ile«HMM_accs[h] .mu[i] [j] [k]«" " ; 
file«"\n"; 
} 

ifCHMM_accs[h].full_cov.Dim()!=0) 
{ 

file«"\nfuU_cov: \n"; 
forCi=l ; i<Nh-l ; 

for(j=0; j<Mh; 

for(k=0 ; k<obs_size ; k++) 

{ 

f or(z=k ; z<obs_si ze ; z++) 
file«HMM_accs 

[h].fuU_cov[i][j][k][z]«" ■ 

file«"\n"; 
} 

} 

else{ 

f i le«"\ti\ndiag_va : \n" ; 
for(i=l;i<Nh-l;i-H-) 
for(j=0; j<Mh; 
{ 

for(k=0;k<obs_size;k++) 

file«HMM_accs[h] .diag_va 



[i][j]Ck]«" 



file«"\n"; 
} 

} 

file«"\nc: \n"; 
forCi=l;i<Nh-l;i++) 
{ 

for(j=0; j<Mh; 

file«HMM_accs[h] .c[i][j]«" " ; 
file«"\n"; 
} 

file«"\n"; 
} 



file.closeO; 
return; 



} 



void ModelsSimultaneousTraining: :Load_Models_Parameters() 
{ 

t_index symbol, num_symbols; 
t_index vec_size; 
String buffer; 
Boolean use_full_cov; 
ifstream init_spcf; 

init_spcf .open(models_file_input, ios: :inlios: :nocreate); 
Read.Data_File_Header (init_spcf, vec_size, use_f uU.cov) ; 

if( features. Feature_Vet_Dim()!=vec_size) 

merr«"Not compatible statistics dimension with initialized 
acoustic models" ; 

Write_Header_Of_File_ModelCmodels_file_output, dbase.Snd_Type(), 
dbase . Label_Type() , dbase . Db_Fi le_Li st_Name() , 
dbase . Window_Lenght() , 

dbase. Window_Overlap(), vec.size, use_full_cov); 

num_symbols = HMM_defs,Dim(); 

HMM_accs . Destroy _And_ReDim(num_symbols) ; 

for(symbol=0; symbol<num_symbols; symbol++) 
{ 

HMM_defs [symbol] . f ile=symbol ; 
HMM_def s [symbol] . stat_dim=vec_si ze ; 
HMM_defs[symbol] .full_covariance=use_full_cov; 
HMM_defs [symbol] ,Read(init_spcf, use_full„cov); 
HMM_accs [symbol] .Configure(HMM_defs 
[symbol] . num_states , HMM_def s [symbol] . num_gauss , 

vec_size, use_full_cov); 

} 

return; 
} 



void ModelsSimultaneousTraining: :Load_Statistic_Accs(const Strings 
accs_file) 
{ 

t_index i , j ,h,k,z,Nh,Mh; 
t.index obs.size, file; 



if St ream file; 
String buffer; 
t_real vol; 

file.open(accs_file,ios: :inl ios: :nocreate); 
ifCfile.failO) 

merr«"Could not open file of statistics accumulators."; 

f i le . precision(OUTPUT_SIZE) ; 

f or(h=0 ; h<HMM_accs . Dim() ; h++) 
{ 

file»buffer; 

file»file; 

Assert(file==h); 

file»buffer ; 

f i le»HMM_accs [h] . num_i stances ; 

file»buffer ; 

file»Nh; 

file»buffer ; 

file»Mh; 

file»buffer; 

for(i=0;i<Nh-l;i++) 

f or( j=l ; j<Nh ; j++) 
{ 

file»val; 

HMM_accs[h] • tran[i] [j]+=val ; 
} 

file»buffer ; 
for(i=0;i<Nh-l;i++) 
{ 

file»val; 

HMM_accs[h] .occ[i]+=val ; 
} 

obs_size=HMM_accs[h] .mu[0] [0] .Dim() ; 
file»buffer ; 

f or(i=l ; i<Nh-l ; i++) 
for(j=0; j<Mh; 

f or(k=0 ; k<obs_size ; k-H-) 
{ 

file»val ; 

HMM_accs[h] .mu[i] [j] [k]+=val ; 
} 



file»buffer; 
if(buffer=="fuU_cov:") 
f or(i=l ; i<Nh-l ; 

for(j=0; j<Mh;j++) 

f or(k=0 ; k<obs_si ze ; k++) 

f or(z=k ; z<obs_si ze ; z++) 
{ 

file»val ; 
HMM.accs 

[h] . f ull.cov[i] [j] [k] [z]+=val ; 

} 

else for(i=l;i<Nh-l;i++) 

for(j=0; j<Mh; 
f or(k=0 ; k<obs_si ze ; k++) 
{ 

file»val; 
HMM_accs[h].diag_vaCi][j][k] 

+=val; 

} 

file»buffer ; 
f or(i=l ; i<Nh-l ; 

for(j=0; j<Mh; 

{ 

file»val; 

HMM_accs[h] . c[i] [j]+=val ; 
} 

} 

file.closeO; 

return; 

} 

// * 

// CLASS EmbNodeSpecShape 

// * 



// MOutP: Returns prob of vector x for given state & given mixture 

t_real EmbNodeSpecShape: :Mix_Obs_LProb(const VetDouble& obs, const t_index 

m) 



{ 

t_real prob; 



prob=(*this)[m] .Evaluate_Exp_Gauss(obs); 
prob+=(*this)[m] .gConst; 

return prob; 
} 

// OutP: Returns probability (log) of vector x for given state 
t_real EmbNodeSpecShape: :Obs_LProb(const VetDouble& obs) 
{ 

t_index m; 
t_real bx,px; 

bx = LOGZERO; /* Multi Mixture Case */ 

for (m=0; m<Dim(); m++) 
{ 

if ((*this)[m].weight>LMINMIX) 
{ 

px =(*this)[m] .Evaluate_Exp_Gauss(obs); 

px += (*this)[m] .gConst; 

bx = LogAdd(bx, (*this)[m] .weight + px); 

} 

} 

return bx; 
} 



